
<!DOCTYPE HTML>
<html lang="" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>2.2019-12-30-spark-RDD · GitBook</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="RDD介绍">
        <meta name="generator" content="GitBook 3.2.3">
        
        
        
    
    <link rel="stylesheet" href="../gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
                
            
        

    

    
        
    
        
    
        
    
        
    
        
    
        
    

        
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">

    
    <link rel="next" href="2019-12-31-hadoop-hdfs命令别名.html" />
    
    
    <link rel="prev" href="2019-12-30-spark-in-docker.html" />
    

    </head>
    <body>
        
<div class="book">
    <div class="book-summary">
        
            
<div id="book-search-input" role="search">
    <input type="text" placeholder="Type to search" />
</div>

            
                <nav role="navigation">
                


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter " data-level="1.1" data-path="../">
            
                <a href="../">
            
                    
                    简介
            
                </a>
            

            
        </li>
    

    
        
        <li class="divider"></li>
        
        
    
        <li class="chapter " data-level="2.1" >
            
                <span>
            
                    
                    [bigdata]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.1.1" data-path="2019-12-30-spark-in-docker.html">
            
                <a href="2019-12-30-spark-in-docker.html">
            
                    
                    1.2019-12-30-spark-in-docker
            
                </a>
            

            
        </li>
    
        <li class="chapter active" data-level="2.1.2" data-path="2019-12-30-spark-RDD.html">
            
                <a href="2019-12-30-spark-RDD.html">
            
                    
                    2.2019-12-30-spark-RDD
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.1.3" data-path="2019-12-31-hadoop-hdfs命令别名.html">
            
                <a href="2019-12-31-hadoop-hdfs命令别名.html">
            
                    
                    3.2019-12-31-hadoop-hdfs命令别名
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.2" >
            
                <span>
            
                    
                    [c]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.2.1" data-path="../c/2020-01-07-c语言.html">
            
                <a href="../c/2020-01-07-c语言.html">
            
                    
                    1.2020-01-07-c语言
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.2.2" data-path="../c/2020-01-07-C语言基础教程.html">
            
                <a href="../c/2020-01-07-C语言基础教程.html">
            
                    
                    2.2020-01-07-C语言基础教程
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.2.3" data-path="../c/2020-01-09-c语言语法基础.html">
            
                <a href="../c/2020-01-09-c语言语法基础.html">
            
                    
                    3.2020-01-09-c语言语法基础
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.3" >
            
                <span>
            
                    
                    [DataStructureandAlgorithms]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.3.1" data-path="../DataStructureandAlgorithms/2019-12-28-数据结构和算法的关系.html">
            
                <a href="../DataStructureandAlgorithms/2019-12-28-数据结构和算法的关系.html">
            
                    
                    1.2019-12-28-数据结构和算法的关系
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.3.2" data-path="../DataStructureandAlgorithms/2019-12-28-稀疏数组.html">
            
                <a href="../DataStructureandAlgorithms/2019-12-28-稀疏数组.html">
            
                    
                    2.2019-12-28-稀疏数组
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.4" >
            
                <span>
            
                    
                    [git]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.4.1" data-path="../git/2019-12-28-git-alias.html">
            
                <a href="../git/2019-12-28-git-alias.html">
            
                    
                    1.2019-12-28-git-alias
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.4.2" data-path="../git/2019-12-29-git常用命令.html">
            
                <a href="../git/2019-12-29-git常用命令.html">
            
                    
                    2.2019-12-29-git常用命令
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.4.3" data-path="../git/2019-12-31-git教程.html">
            
                <a href="../git/2019-12-31-git教程.html">
            
                    
                    3.2019-12-31-git教程
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.5" >
            
                <span>
            
                    
                    [go]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.5.1" data-path="../go/2019-12-30-beego基本使用.html">
            
                <a href="../go/2019-12-30-beego基本使用.html">
            
                    
                    1.2019-12-30-beego基本使用
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.6" >
            
                <span>
            
                    
                    [java]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.6.1" data-path="../java/2019-12-30-环境变量和系统属性.html">
            
                <a href="../java/2019-12-30-环境变量和系统属性.html">
            
                    
                    1.2019-12-30-环境变量和系统属性
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.2" data-path="../java/2019-12-31-cloud-native-云原生.html">
            
                <a href="../java/2019-12-31-cloud-native-云原生.html">
            
                    
                    2.2019-12-31-cloud-native-云原生
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.3" data-path="../java/2019-12-31-java语法难点.html">
            
                <a href="../java/2019-12-31-java语法难点.html">
            
                    
                    3.2019-12-31-java语法难点
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.4" data-path="../java/2019-12-31-设计模式.html">
            
                <a href="../java/2019-12-31-设计模式.html">
            
                    
                    4.2019-12-31-设计模式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.5" data-path="../java/2020-01-12-Ajax跨域问题解决方案.html">
            
                <a href="../java/2020-01-12-Ajax跨域问题解决方案.html">
            
                    
                    5.2020-01-12-Ajax跨域问题解决方案
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.6" data-path="../java/2020-01-12-freemarker应用场景介绍.html">
            
                <a href="../java/2020-01-12-freemarker应用场景介绍.html">
            
                    
                    6.2020-01-12-freemarker应用场景介绍
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.7" data-path="../java/2020-01-12-jsp运行原理和运行过程.html">
            
                <a href="../java/2020-01-12-jsp运行原理和运行过程.html">
            
                    
                    7.2020-01-12-jsp运行原理和运行过程
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.8" data-path="../java/2020-01-12-深入理解Maven中的Profile.html">
            
                <a href="../java/2020-01-12-深入理解Maven中的Profile.html">
            
                    
                    8.2020-01-12-深入理解Maven中的Profile
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.9" data-path="../java/2020-01-12-缓存穿透-缓存击穿-缓存雪崩概念及解决方案.html">
            
                <a href="../java/2020-01-12-缓存穿透-缓存击穿-缓存雪崩概念及解决方案.html">
            
                    
                    9.2020-01-12-缓存穿透-缓存击穿-缓存雪崩概念及解决方案
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.10" data-path="../java/2020-01-13-事务笔记.html">
            
                <a href="../java/2020-01-13-事务笔记.html">
            
                    
                    10.2020-01-13-事务笔记
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.11" data-path="../java/2020-01-13-幂等性.html">
            
                <a href="../java/2020-01-13-幂等性.html">
            
                    
                    11.2020-01-13-幂等性
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.6.12" data-path="../java/2020-01-16-java基本语法.html">
            
                <a href="../java/2020-01-16-java基本语法.html">
            
                    
                    12.2020-01-16-java基本语法
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.7" >
            
                <span>
            
                    
                    [arthas]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.7.1" data-path="../java/arthas/2019-12-28-arthas-example.html">
            
                <a href="../java/arthas/2019-12-28-arthas-example.html">
            
                    
                    1.2019-12-28-arthas-example
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.2" data-path="../java/arthas/2019-12-28-arthas添加支持执行历史命令的shell功能.html">
            
                <a href="../java/arthas/2019-12-28-arthas添加支持执行历史命令的shell功能.html">
            
                    
                    2.2019-12-28-arthas添加支持执行历史命令的shell功能
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.3" data-path="../java/arthas/2020-01-10-思考性能调优.html">
            
                <a href="../java/arthas/2020-01-10-思考性能调优.html">
            
                    
                    3.2020-01-10-思考性能调优
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.4" data-path="../java/arthas/2020-01-10-性能调优举例-最佳实践.html">
            
                <a href="../java/arthas/2020-01-10-性能调优举例-最佳实践.html">
            
                    
                    4.2020-01-10-性能调优举例-最佳实践
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.5" data-path="../java/arthas/2020-01-10-性能调优举例一.html">
            
                <a href="../java/arthas/2020-01-10-性能调优举例一.html">
            
                    
                    5.2020-01-10-性能调优举例一
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.6" data-path="../java/arthas/2020-01-10-性能调优举例三.html">
            
                <a href="../java/arthas/2020-01-10-性能调优举例三.html">
            
                    
                    6.2020-01-10-性能调优举例三
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.7" data-path="../java/arthas/2020-01-10-性能调优举例二.html">
            
                <a href="../java/arthas/2020-01-10-性能调优举例二.html">
            
                    
                    7.2020-01-10-性能调优举例二
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.8" data-path="../java/arthas/2020-01-10-性能调优举例四.html">
            
                <a href="../java/arthas/2020-01-10-性能调优举例四.html">
            
                    
                    8.2020-01-10-性能调优举例四
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.9" data-path="../java/arthas/2020-01-15-性能调优举例五.html">
            
                <a href="../java/arthas/2020-01-15-性能调优举例五.html">
            
                    
                    9.2020-01-15-性能调优举例五
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.10" data-path="../java/arthas/2020-05-12-性能调优举例六.html">
            
                <a href="../java/arthas/2020-05-12-性能调优举例六.html">
            
                    
                    10.2020-05-12-性能调优举例六
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.7.11" data-path="../java/arthas/arthas-mind-map.html">
            
                <a href="../java/arthas/arthas-mind-map.html">
            
                    
                    11.arthas-mind-map
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.8" >
            
                <span>
            
                    
                    [dubbo]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.8.1" data-path="../java/dubbo/2019-12-28-dubbo涉及的技术.html">
            
                <a href="../java/dubbo/2019-12-28-dubbo涉及的技术.html">
            
                    
                    1.2019-12-28-dubbo涉及的技术
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.8.2" data-path="../java/dubbo/2019-12-30-dubbo消费者访问流程.html">
            
                <a href="../java/dubbo/2019-12-30-dubbo消费者访问流程.html">
            
                    
                    2.2019-12-30-dubbo消费者访问流程
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.8.3" data-path="../java/dubbo/2019-12-30-dubbo面试.html">
            
                <a href="../java/dubbo/2019-12-30-dubbo面试.html">
            
                    
                    3.2019-12-30-dubbo面试
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.9" >
            
                <span>
            
                    
                    [jdk]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.9.1" data-path="../java/jdk/2020-01-12-实例介绍 JAVA 泛型基础.html">
            
                <a href="../java/jdk/2020-01-12-实例介绍 JAVA 泛型基础.html">
            
                    
                    1.2020-01-12-实例介绍 JAVA 泛型基础
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.9.2" data-path="../java/jdk/2020-01-13-ThreadLocal六连问.html">
            
                <a href="../java/jdk/2020-01-13-ThreadLocal六连问.html">
            
                    
                    2.2020-01-13-ThreadLocal六连问
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.10" >
            
                <span>
            
                    
                    [jvm]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.10.1" data-path="../java/jvm/2019-12-30-性能调优.html">
            
                <a href="../java/jvm/2019-12-30-性能调优.html">
            
                    
                    1.2019-12-30-性能调优
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.11" >
            
                <span>
            
                    
                    [netty]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.11.1" data-path="../java/netty/2020-01-16-netty入门.html">
            
                <a href="../java/netty/2020-01-16-netty入门.html">
            
                    
                    1.2020-01-16-netty入门
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.11.2" data-path="../java/netty/2020-01-16-netty核心概念.html">
            
                <a href="../java/netty/2020-01-16-netty核心概念.html">
            
                    
                    2.2020-01-16-netty核心概念
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.12" >
            
                <span>
            
                    
                    [solr]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.12.1" data-path="../java/solr/2019-12-28-solr-7.6.0.tgz安装.html">
            
                <a href="../java/solr/2019-12-28-solr-7.6.0.tgz安装.html">
            
                    
                    1.2019-12-28-solr-7.6.0.tgz安装
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.13" >
            
                <span>
            
                    
                    [spring]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.13.1" data-path="../java/spring/2020-01-12-SpringBoot属性读取方式介绍.html">
            
                <a href="../java/spring/2020-01-12-SpringBoot属性读取方式介绍.html">
            
                    
                    1.2020-01-12-SpringBoot属性读取方式介绍
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.13.2" data-path="../java/spring/2020-01-12-使用spring boot+mybatis plus快速入门.html">
            
                <a href="../java/spring/2020-01-12-使用spring boot+mybatis plus快速入门.html">
            
                    
                    2.2020-01-12-使用spring boot+mybatis plus快速入门
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.13.3" data-path="../java/spring/2020-01-13-spring-initMethod-destoryMethod.html">
            
                <a href="../java/spring/2020-01-13-spring-initMethod-destoryMethod.html">
            
                    
                    3.2020-01-13-spring-initMethod-destoryMethod
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.13.4" data-path="../java/spring/2020-01-13-spring循环依赖.html">
            
                <a href="../java/spring/2020-01-13-spring循环依赖.html">
            
                    
                    4.2020-01-13-spring循环依赖
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.13.5" data-path="../java/spring/2020-01-13-spring源码随记.html">
            
                <a href="../java/spring/2020-01-13-spring源码随记.html">
            
                    
                    5.2020-01-13-spring源码随记
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.13.6" data-path="../java/spring/2020-01-13-自定义mini-spring.html">
            
                <a href="../java/spring/2020-01-13-自定义mini-spring.html">
            
                    
                    6.2020-01-13-自定义mini-spring
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.14" >
            
                <span>
            
                    
                    [websocket]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.14.1" data-path="../java/websocket/2019-12-30-webscoket-springboot通信.html">
            
                <a href="../java/websocket/2019-12-30-webscoket-springboot通信.html">
            
                    
                    1.2019-12-30-webscoket-springboot通信
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.15" >
            
                <span>
            
                    
                    [linux]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.15.1" data-path="../linux/2019-12-28-Linux常用命令和快捷键.html">
            
                <a href="../linux/2019-12-28-Linux常用命令和快捷键.html">
            
                    
                    1.2019-12-28-Linux常用命令和快捷键
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.2" data-path="../linux/2019-12-28-从零到一环境安装.html">
            
                <a href="../linux/2019-12-28-从零到一环境安装.html">
            
                    
                    2.2019-12-28-从零到一环境安装
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.3" data-path="../linux/2019-12-30-vi-常用命令.html">
            
                <a href="../linux/2019-12-30-vi-常用命令.html">
            
                    
                    3.2019-12-30-vi-常用命令
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.4" data-path="../linux/2019-12-31-awk命令.html">
            
                <a href="../linux/2019-12-31-awk命令.html">
            
                    
                    4.2019-12-31-awk命令
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.5" data-path="../linux/2019-12-31-cup信息分析.html">
            
                <a href="../linux/2019-12-31-cup信息分析.html">
            
                    
                    5.2019-12-31-cup信息分析
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.6" data-path="../linux/2019-12-31-shell编程.html">
            
                <a href="../linux/2019-12-31-shell编程.html">
            
                    
                    6.2019-12-31-shell编程
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.7" data-path="../linux/2019-12-31-top命令.html">
            
                <a href="../linux/2019-12-31-top命令.html">
            
                    
                    7.2019-12-31-top命令
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.8" data-path="../linux/2019-12-31-普通用户docker的问题.html">
            
                <a href="../linux/2019-12-31-普通用户docker的问题.html">
            
                    
                    8.2019-12-31-普通用户docker的问题
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.9" data-path="../linux/2020-01-01-centos7 安装kubernetes.html">
            
                <a href="../linux/2020-01-01-centos7 安装kubernetes.html">
            
                    
                    9.2020-01-01-centos7 安装kubernetes
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.10" data-path="../linux/2020-01-01-centos安装常用配置.html">
            
                <a href="../linux/2020-01-01-centos安装常用配置.html">
            
                    
                    10.2020-01-01-centos安装常用配置
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.11" data-path="../linux/2020-01-01-docker使用.html">
            
                <a href="../linux/2020-01-01-docker使用.html">
            
                    
                    11.2020-01-01-docker使用
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.12" data-path="../linux/2020-01-01-kubernetes使用.html">
            
                <a href="../linux/2020-01-01-kubernetes使用.html">
            
                    
                    12.2020-01-01-kubernetes使用
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.13" data-path="../linux/2020-05-12-PG有用的SQL.html">
            
                <a href="../linux/2020-05-12-PG有用的SQL.html">
            
                    
                    13.2020-05-12-PG有用的SQL
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.15.14" data-path="../linux/2020-05-12-vim命令.html">
            
                <a href="../linux/2020-05-12-vim命令.html">
            
                    
                    14.2020-05-12-vim命令
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.16" >
            
                <span>
            
                    
                    [automate]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.16.1" data-path="../linux/automate/2020-01-12-redis伪集群创建脚本.html">
            
                <a href="../linux/automate/2020-01-12-redis伪集群创建脚本.html">
            
                    
                    1.2020-01-12-redis伪集群创建脚本
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.16.2" data-path="../linux/automate/2020-01-12-tomcat集群创建脚本.html">
            
                <a href="../linux/automate/2020-01-12-tomcat集群创建脚本.html">
            
                    
                    2.2020-01-12-tomcat集群创建脚本
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.16.3" data-path="../linux/automate/2020-01-12-zookeeper自动化脚本.html">
            
                <a href="../linux/automate/2020-01-12-zookeeper自动化脚本.html">
            
                    
                    3.2020-01-12-zookeeper自动化脚本
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.16.4" data-path="../linux/automate/2020-01-12-其他自动化脚本.html">
            
                <a href="../linux/automate/2020-01-12-其他自动化脚本.html">
            
                    
                    4.2020-01-12-其他自动化脚本
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.17" >
            
                <span>
            
                    
                    [MySQL]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.17.1" data-path="../MySQL/2019-12-31-mysql常用命令.html">
            
                <a href="../MySQL/2019-12-31-mysql常用命令.html">
            
                    
                    1.2019-12-31-mysql常用命令
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.18" >
            
                <span>
            
                    
                    [nginx]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.18.1" data-path="../nginx/2019-12-29-Nginx-Keepalived高可用集群.html">
            
                <a href="../nginx/2019-12-29-Nginx-Keepalived高可用集群.html">
            
                    
                    1.2019-12-29-Nginx-Keepalived高可用集群
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.18.2" data-path="../nginx/2020-01-16-nginx参数详细说明.html">
            
                <a href="../nginx/2020-01-16-nginx参数详细说明.html">
            
                    
                    2.2020-01-16-nginx参数详细说明
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.19" >
            
                <span>
            
                    
                    [other]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.19.1" data-path="../other/2019-12-28-知识小计.html">
            
                <a href="../other/2019-12-28-知识小计.html">
            
                    
                    1.2019-12-28-知识小计
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.19.2" data-path="../other/2020-01-04-科学上网.html">
            
                <a href="../other/2020-01-04-科学上网.html">
            
                    
                    2.2020-01-04-科学上网
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.19.3" data-path="../other/2020-05-13-正则表达式.html">
            
                <a href="../other/2020-05-13-正则表达式.html">
            
                    
                    3.2020-05-13-正则表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.19.4" data-path="../other/template.html">
            
                <a href="../other/template.html">
            
                    
                    4.template
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.20" >
            
                <span>
            
                    
                    [PostgreSQL]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.20.1" data-path="../PostgreSQL/2019-12-28-PostgreSQL安装文档.html">
            
                <a href="../PostgreSQL/2019-12-28-PostgreSQL安装文档.html">
            
                    
                    1.2019-12-28-PostgreSQL安装文档
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.20.2" data-path="../PostgreSQL/2019-12-28-Postgres学习笔记.html">
            
                <a href="../PostgreSQL/2019-12-28-Postgres学习笔记.html">
            
                    
                    2.2019-12-28-Postgres学习笔记
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.20.3" data-path="../PostgreSQL/2019-12-28-脚本MYSQL导出数据到PG.html">
            
                <a href="../PostgreSQL/2019-12-28-脚本MYSQL导出数据到PG.html">
            
                    
                    3.2019-12-28-脚本MYSQL导出数据到PG
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.20.4" data-path="../PostgreSQL/2020-01-04-docker-PostgreSQL安装文档.html">
            
                <a href="../PostgreSQL/2020-01-04-docker-PostgreSQL安装文档.html">
            
                    
                    4.2020-01-04-docker-PostgreSQL安装文档
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.20.5" data-path="../PostgreSQL/2020-01-17-阿里云PG主从切换问题.html">
            
                <a href="../PostgreSQL/2020-01-17-阿里云PG主从切换问题.html">
            
                    
                    5.2020-01-17-阿里云PG主从切换问题
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.21" >
            
                <span>
            
                    
                    [python]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.21.1" data-path="../python/2019-12-28-pip-install指定国内源镜像.html">
            
                <a href="../python/2019-12-28-pip-install指定国内源镜像.html">
            
                    
                    1.2019-12-28-pip-install指定国内源镜像
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.21.2" data-path="../python/2019-12-30-python内置函数.html">
            
                <a href="../python/2019-12-30-python内置函数.html">
            
                    
                    2.2019-12-30-python内置函数
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.21.3" data-path="../python/2019-12-30-一图搞懂基本python语法 .html">
            
                <a href="../python/2019-12-30-一图搞懂基本python语法 .html">
            
                    
                    3.2019-12-30-一图搞懂基本python语法 
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.22" >
            
                <span>
            
                    
                    [redis]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.22.1" data-path="../redis/2019-12-28-redis5.0.3安装.html">
            
                <a href="../redis/2019-12-28-redis5.0.3安装.html">
            
                    
                    1.2019-12-28-redis5.0.3安装
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.22.2" data-path="../redis/2019-12-28-redis5.0.3集群搭建.html">
            
                <a href="../redis/2019-12-28-redis5.0.3集群搭建.html">
            
                    
                    2.2019-12-28-redis5.0.3集群搭建
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.23" >
            
                <span>
            
                    
                    [tool]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.23.1" data-path="../tool/2019-12-28-idea高效使用技巧和快捷键.html">
            
                <a href="../tool/2019-12-28-idea高效使用技巧和快捷键.html">
            
                    
                    1.2019-12-28-idea高效使用技巧和快捷键
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.23.2" data-path="../tool/2019-12-28-PlantUML画图脚本.html">
            
                <a href="../tool/2019-12-28-PlantUML画图脚本.html">
            
                    
                    2.2019-12-28-PlantUML画图脚本
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.23.3" data-path="../tool/2019-12-30-vscode高效使用.html">
            
                <a href="../tool/2019-12-30-vscode高效使用.html">
            
                    
                    3.2019-12-30-vscode高效使用
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2.24" >
            
                <span>
            
                    
                    [windows]
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.24.1" data-path="../windows/2020-01-12-本地服务成为互联网服务.html">
            
                <a href="../windows/2020-01-12-本地服务成为互联网服务.html">
            
                    
                    1.2020-01-12-本地服务成为互联网服务
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="2.24.2" data-path="../windows/2020-01-12-注册bat为windows服务.html">
            
                <a href="../windows/2020-01-12-注册bat为windows服务.html">
            
                    
                    2.2020-01-12-注册bat为windows服务
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            Published with GitBook
        </a>
    </li>
</ul>


                </nav>
            
        
    </div>

    <div class="book-body">
        
            <div class="body-inner">
                
                    

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href=".." >2.2019-12-30-spark-RDD</a>
    </h1>
</div>




                    <div class="page-wrapper" tabindex="-1" role="main">
                        <div class="page-inner">
                            
<div id="book-search-results">
    <div class="search-noresults">
    
                                <section class="normal markdown-section">
                                
                                <p>RDD&#x4ECB;&#x7ECD;</p>
<h2 id="11rdd&#x4E3A;&#x4EC0;&#x4E48;&#x4F1A;&#x4EA7;&#x751F;">1.1RDD&#x4E3A;&#x4EC0;&#x4E48;&#x4F1A;&#x4EA7;&#x751F;</h2>
<p>RDD&#x662F;Spark&#x7684;&#x57FA;&#x77F3;&#xFF0C;&#x662F;&#x5B9E;&#x73B0;Spark&#x6570;&#x636E;&#x5904;&#x7406;&#x7684;&#x6838;&#x5FC3;&#x62BD;&#x8C61;&#x3002;&#x90A3;&#x4E48;RDD&#x4E3A;&#x4EC0;&#x4E48;&#x4F1A;&#x4EA7;&#x751F;&#x5462;&#xFF1F;
Hadoop&#x7684;MapReduce&#x662F;&#x4E00;&#x79CD;&#x57FA;&#x4E8E;&#x6570;&#x636E;&#x96C6;&#x7684;&#x5DE5;&#x4F5C;&#x6A21;&#x5F0F;&#xFF0C;&#x9762;&#x5411;&#x6570;&#x636E;&#xFF0C;&#x8FD9;&#x79CD;&#x5DE5;&#x4F5C;&#x6A21;&#x5F0F;&#x4E00;&#x822C;&#x662F;&#x4ECE;&#x5B58;&#x50A8;&#x4E0A;&#x52A0;&#x8F7D;&#x6570;&#x636E;&#x96C6;&#xFF0C;&#x7136;&#x540E;&#x64CD;&#x4F5C;&#x6570;&#x636E;&#x96C6;&#xFF0C;&#x6700;&#x540E;&#x5199;&#x5165;&#x7269;&#x7406;&#x5B58;&#x50A8;&#x8BBE;&#x5907;&#x3002;&#x6570;&#x636E;&#x66F4;&#x591A;&#x9762;&#x4E34;&#x7684;&#x662F;&#x4E00;&#x6B21;&#x6027;&#x5904;&#x7406;&#x3002;
MR&#x7684;&#x8FD9;&#x79CD;&#x65B9;&#x5F0F;&#x5BF9;&#x6570;&#x636E;&#x9886;&#x57DF;&#x4E24;&#x79CD;&#x5E38;&#x89C1;&#x7684;&#x64CD;&#x4F5C;&#x4E0D;&#x662F;&#x5F88;&#x9AD8;&#x6548;&#x3002;&#x7B2C;&#x4E00;&#x79CD;&#x662F;&#x8FED;&#x4EE3;&#x5F0F;&#x7684;&#x7B97;&#x6CD5;&#x3002;&#x6BD4;&#x5982;&#x673A;&#x5668;&#x5B66;&#x4E60;&#x4E2D;ALS&#x3001;&#x51F8;&#x4F18;&#x5316;&#x68AF;&#x5EA6;&#x4E0B;&#x964D;&#x7B49;&#x3002;&#x8FD9;&#x4E9B;&#x90FD;&#x9700;&#x8981;&#x57FA;&#x4E8E;&#x6570;&#x636E;&#x96C6;&#x6216;&#x8005;&#x6570;&#x636E;&#x96C6;&#x7684;&#x884D;&#x751F;&#x6570;&#x636E;&#x53CD;&#x590D;&#x67E5;&#x8BE2;&#x53CD;&#x590D;&#x64CD;&#x4F5C;&#x3002;MR&#x8FD9;&#x79CD;&#x6A21;&#x5F0F;&#x4E0D;&#x592A;&#x5408;&#x9002;&#xFF0C;&#x5373;&#x4F7F;&#x591A;MR&#x4E32;&#x884C;&#x5904;&#x7406;&#xFF0C;&#x6027;&#x80FD;&#x548C;&#x65F6;&#x95F4;&#x4E5F;&#x662F;&#x4E00;&#x4E2A;&#x95EE;&#x9898;&#x3002;&#x6570;&#x636E;&#x7684;&#x5171;&#x4EAB;&#x4F9D;&#x8D56;&#x4E8E;&#x78C1;&#x76D8;&#x3002;&#x53E6;&#x5916;&#x4E00;&#x79CD;&#x662F;&#x4EA4;&#x4E92;&#x5F0F;&#x6570;&#x636E;&#x6316;&#x6398;&#xFF0C;MR&#x663E;&#x7136;&#x4E0D;&#x64C5;&#x957F;&#x3002;
&#x6211;&#x4EEC;&#x9700;&#x8981;&#x4E00;&#x4E2A;&#x6548;&#x7387;&#x975E;&#x5E38;&#x5FEB;&#xFF0C;&#x4E14;&#x80FD;&#x591F;&#x652F;&#x6301;&#x8FED;&#x4EE3;&#x8BA1;&#x7B97;&#x548C;&#x6709;&#x6548;&#x6570;&#x636E;&#x5171;&#x4EAB;&#x7684;&#x6A21;&#x578B;&#xFF0C;Spark&#x5E94;&#x8FD0;&#x800C;&#x751F;&#x3002;RDD&#x662F;&#x57FA;&#x4E8E;&#x5DE5;&#x4F5C;&#x96C6;&#x7684;&#x5DE5;&#x4F5C;&#x6A21;&#x5F0F;&#xFF0C;&#x66F4;&#x591A;&#x7684;&#x662F;&#x9762;&#x5411;&#x5DE5;&#x4F5C;&#x6D41;&#x3002;
&#x4F46;&#x662F;&#x65E0;&#x8BBA;&#x662F;MR&#x8FD8;&#x662F;RDD&#x90FD;&#x5E94;&#x8BE5;&#x5177;&#x6709;&#x7C7B;&#x4F3C;&#x4F4D;&#x7F6E;&#x611F;&#x77E5;&#x3001;&#x5BB9;&#x9519;&#x548C;&#x8D1F;&#x8F7D;&#x5747;&#x8861;&#x7B49;&#x7279;&#x6027;&#x3002;</p>
<h2 id="12rdd&#x6982;&#x8FF0;">1.2RDD&#x6982;&#x8FF0;</h2>
<h3 id="121&#x4EC0;&#x4E48;&#x662F;rdd">1.2.1&#x4EC0;&#x4E48;&#x662F;RDD</h3>
<ol>
<li>RDD&#xFF08;Resilient Distributed Dataset&#xFF09;&#x53EB;&#x505A;&#x5F39;&#x6027;&#x5206;&#x5E03;&#x5F0F;&#x6570;&#x636E;&#x96C6;&#xFF0C;&#x662F;Spark&#x4E2D;&#x6700;&#x57FA;&#x672C;&#x7684;&#x6570;&#x636E;&#x62BD;&#x8C61;&#xFF0C;&#x5B83;&#x4EE3;&#x8868;&#x4E00;&#x4E2A;&#x4E0D;&#x53EF;&#x53D8;&#x3001;&#x53EF;&#x5206;&#x533A;&#x3001;&#x91CC;&#x9762;&#x7684;&#x5143;&#x7D20;&#x53EF;&#x5E76;&#x884C;&#x8BA1;&#x7B97;&#x7684;&#x96C6;&#x5408;&#x3002;&#x5728; Spark &#x4E2D;&#xFF0C;&#x5BF9;&#x6570;&#x636E;&#x7684;&#x6240;&#x6709;&#x64CD;&#x4F5C;&#x4E0D;&#x5916;&#x4E4E;&#x521B;&#x5EFA; RDD&#x3001;&#x8F6C;&#x5316;&#x5DF2;&#x6709;RDD &#x4EE5;&#x53CA;&#x8C03;&#x7528; RDD &#x64CD;&#x4F5C;&#x8FDB;&#x884C;&#x6C42;&#x503C;&#x3002;&#x6BCF;&#x4E2A; RDD &#x90FD;&#x88AB;&#x5206;&#x4E3A;&#x591A;&#x4E2A;&#x5206;&#x533A;&#xFF0C;&#x8FD9;&#x4E9B;&#x5206;&#x533A;&#x8FD0;&#x884C;&#x5728;&#x96C6;&#x7FA4;&#x4E2D;&#x7684;&#x4E0D;&#x540C;&#x8282;&#x70B9;&#x4E0A;&#x3002;RDD &#x53EF;&#x4EE5;&#x5305;&#x542B; Python&#x3001;Java&#x3001;Scala &#x4E2D;&#x4EFB;&#x610F;&#x7C7B;&#x578B;&#x7684;&#x5BF9;&#x8C61;&#xFF0C; &#x751A;&#x81F3;&#x53EF;&#x4EE5;&#x5305;&#x542B;&#x7528;&#x6237;&#x81EA;&#x5B9A;&#x4E49;&#x7684;&#x5BF9;&#x8C61;&#x3002;RDD&#x5177;&#x6709;&#x6570;&#x636E;&#x6D41;&#x6A21;&#x578B;&#x7684;&#x7279;&#x70B9;&#xFF1A;&#x81EA;&#x52A8;&#x5BB9;&#x9519;&#x3001;&#x4F4D;&#x7F6E;&#x611F;&#x77E5;&#x6027;&#x8C03;&#x5EA6;&#x548C;&#x53EF;&#x4F38;&#x7F29;&#x6027;&#x3002;RDD&#x5141;&#x8BB8;&#x7528;&#x6237;&#x5728;&#x6267;&#x884C;&#x591A;&#x4E2A;&#x67E5;&#x8BE2;&#x65F6;&#x663E;&#x5F0F;&#x5730;&#x5C06;&#x5DE5;&#x4F5C;&#x96C6;&#x7F13;&#x5B58;&#x5728;&#x5185;&#x5B58;&#x4E2D;&#xFF0C;&#x540E;&#x7EED;&#x7684;&#x67E5;&#x8BE2;&#x80FD;&#x591F;&#x91CD;&#x7528;&#x5DE5;&#x4F5C;&#x96C6;&#xFF0C;&#x8FD9;&#x6781;&#x5927;&#x5730;&#x63D0;&#x5347;&#x4E86;&#x67E5;&#x8BE2;&#x901F;&#x5EA6;&#x3002;</li>
<li>RDD&#x652F;&#x6301;&#x4E24;&#x79CD;&#x64CD;&#x4F5C;:&#x8F6C;&#x5316;&#x64CD;&#x4F5C;&#x548C;&#x884C;&#x52A8;&#x64CD;&#x4F5C;&#x3002;RDD &#x7684;&#x8F6C;&#x5316;&#x64CD;&#x4F5C;&#x662F;&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x65B0;&#x7684; RDD&#x7684;&#x64CD;&#x4F5C;&#xFF0C;&#x6BD4;&#x5982; map()&#x548C; filter()&#xFF0C;&#x800C;&#x884C;&#x52A8;&#x64CD;&#x4F5C;&#x5219;&#x662F;&#x5411;&#x9A71;&#x52A8;&#x5668;&#x7A0B;&#x5E8F;&#x8FD4;&#x56DE;&#x7ED3;&#x679C;&#x6216;&#x628A;&#x7ED3;&#x679C;&#x5199;&#x5165;&#x5916;&#x90E8;&#x7CFB;&#x7EDF;&#x7684;&#x64CD;&#x4F5C;&#x3002;&#x6BD4;&#x5982; count() &#x548C; first()&#x3002; </li>
<li>Spark&#x91C7;&#x7528;&#x60F0;&#x6027;&#x8BA1;&#x7B97;&#x6A21;&#x5F0F;&#xFF0C;RDD&#x53EA;&#x6709;&#x7B2C;&#x4E00;&#x6B21;&#x5728;&#x4E00;&#x4E2A;&#x884C;&#x52A8;&#x64CD;&#x4F5C;&#x4E2D;&#x7528;&#x5230;&#x65F6;&#xFF0C;&#x624D;&#x4F1A;&#x771F;&#x6B63;&#x8BA1;&#x7B97;&#x3002;Spark&#x53EF;&#x4EE5;&#x4F18;&#x5316;&#x6574;&#x4E2A;&#x8BA1;&#x7B97;&#x8FC7;&#x7A0B;&#x3002;&#x9ED8;&#x8BA4;&#x60C5;&#x51B5;&#x4E0B;&#xFF0C;Spark &#x7684; RDD &#x4F1A;&#x5728;&#x4F60;&#x6BCF;&#x6B21;&#x5BF9;&#x5B83;&#x4EEC;&#x8FDB;&#x884C;&#x884C;&#x52A8;&#x64CD;&#x4F5C;&#x65F6;&#x91CD;&#x65B0;&#x8BA1;&#x7B97;&#x3002;&#x5982;&#x679C;&#x60F3;&#x5728;&#x591A;&#x4E2A;&#x884C;&#x52A8;&#x64CD;&#x4F5C;&#x4E2D;&#x91CD;&#x7528;&#x540C;&#x4E00;&#x4E2A; RDD&#xFF0C;&#x53EF;&#x4EE5;&#x4F7F;&#x7528; RDD.persist() &#x8BA9; Spark &#x628A;&#x8FD9;&#x4E2A; RDD &#x7F13;&#x5B58;&#x4E0B;&#x6765;&#x3002;</li>
</ol>
<h3 id="122rdd&#x7684;&#x5C5E;&#x6027;">1.2.2RDD&#x7684;&#x5C5E;&#x6027;</h3>
<ul>
<li><p>1)&#x4E00;&#x7EC4;&#x5206;&#x7247;&#xFF08;Partition&#xFF09;&#xFF0C;&#x5373;&#x6570;&#x636E;&#x96C6;&#x7684;&#x57FA;&#x672C;&#x7EC4;&#x6210;&#x5355;&#x4F4D;&#x3002;&#x5BF9;&#x4E8E;RDD&#x6765;&#x8BF4;&#xFF0C;&#x6BCF;&#x4E2A;&#x5206;&#x7247;&#x90FD;&#x4F1A;&#x88AB;&#x4E00;&#x4E2A;&#x8BA1;&#x7B97;&#x4EFB;&#x52A1;&#x5904;&#x7406;&#xFF0C;&#x5E76;&#x51B3;&#x5B9A;&#x5E76;&#x884C;&#x8BA1;&#x7B97;&#x7684;&#x7C92;&#x5EA6;&#x3002;&#x7528;&#x6237;&#x53EF;&#x4EE5;&#x5728;&#x521B;&#x5EFA;RDD&#x65F6;&#x6307;&#x5B9A;RDD&#x7684;&#x5206;&#x7247;&#x4E2A;&#x6570;&#xFF0C;&#x5982;&#x679C;&#x6CA1;&#x6709;&#x6307;&#x5B9A;&#xFF0C;&#x90A3;&#x4E48;&#x5C31;&#x4F1A;&#x91C7;&#x7528;&#x9ED8;&#x8BA4;&#x503C;&#x3002;&#x9ED8;&#x8BA4;&#x503C;&#x5C31;&#x662F;&#x7A0B;&#x5E8F;&#x6240;&#x5206;&#x914D;&#x5230;&#x7684;CPU Core&#x7684;&#x6570;&#x76EE;&#x3002;</p>
</li>
<li><p>2)&#x4E00;&#x4E2A;&#x8BA1;&#x7B97;&#x6BCF;&#x4E2A;&#x5206;&#x533A;&#x7684;&#x51FD;&#x6570;&#x3002;Spark&#x4E2D;RDD&#x7684;&#x8BA1;&#x7B97;&#x662F;&#x4EE5;&#x5206;&#x7247;&#x4E3A;&#x5355;&#x4F4D;&#x7684;&#xFF0C;&#x6BCF;&#x4E2A;RDD&#x90FD;&#x4F1A;&#x5B9E;&#x73B0;compute&#x51FD;&#x6570;&#x4EE5;&#x8FBE;&#x5230;&#x8FD9;&#x4E2A;&#x76EE;&#x7684;&#x3002;compute&#x51FD;&#x6570;&#x4F1A;&#x5BF9;&#x8FED;&#x4EE3;&#x5668;&#x8FDB;&#x884C;&#x590D;&#x5408;&#xFF0C;&#x4E0D;&#x9700;&#x8981;&#x4FDD;&#x5B58;&#x6BCF;&#x6B21;&#x8BA1;&#x7B97;&#x7684;&#x7ED3;&#x679C;&#x3002;</p>
</li>
<li><p>3)&#x4E00;&#x4E2A;&#x5217;&#x8868;,&#x8868;&#x793A;RDD&#x4E4B;&#x95F4;&#x7684;&#x4F9D;&#x8D56;&#x5173;&#x7CFB;&#x3002;RDD&#x7684;&#x6BCF;&#x6B21;&#x8F6C;&#x6362;&#x90FD;&#x4F1A;&#x751F;&#x6210;&#x4E00;&#x4E2A;&#x65B0;&#x7684;RDD&#xFF0C;&#x6240;&#x4EE5;RDD&#x4E4B;&#x95F4;&#x5C31;&#x4F1A;&#x5F62;&#x6210;&#x7C7B;&#x4F3C;&#x4E8E;&#x6D41;&#x6C34;&#x7EBF;&#x4E00;&#x6837;&#x7684;&#x524D;&#x540E;&#x4F9D;&#x8D56;&#x5173;&#x7CFB;&#x3002;&#x5728;&#x90E8;&#x5206;&#x5206;&#x533A;&#x6570;&#x636E;&#x4E22;&#x5931;&#x65F6;&#xFF0C;Spark&#x53EF;&#x4EE5;&#x901A;&#x8FC7;&#x8FD9;&#x4E2A;&#x4F9D;&#x8D56;&#x5173;&#x7CFB;&#x91CD;&#x65B0;&#x8BA1;&#x7B97;&#x4E22;&#x5931;&#x7684;&#x5206;&#x533A;&#x6570;&#x636E;&#xFF0C;&#x800C;&#x4E0D;&#x662F;&#x5BF9;RDD&#x7684;&#x6240;&#x6709;&#x5206;&#x533A;&#x8FDB;&#x884C;&#x91CD;&#x65B0;&#x8BA1;&#x7B97;&#x3002;</p>
</li>
<li><p>4)&#x4E00;&#x4E2A;Partitioner&#xFF0C;&#x5373;RDD&#x7684;&#x5206;&#x7247;&#x51FD;&#x6570;&#x3002;&#x5F53;&#x524D;Spark&#x4E2D;&#x5B9E;&#x73B0;&#x4E86;&#x4E24;&#x79CD;&#x7C7B;&#x578B;&#x7684;&#x5206;&#x7247;&#x51FD;&#x6570;&#xFF0C;&#x4E00;&#x4E2A;&#x662F;&#x57FA;&#x4E8E;&#x54C8;&#x5E0C;&#x7684;HashPartitioner&#xFF0C;&#x53E6;&#x5916;&#x4E00;&#x4E2A;&#x662F;&#x57FA;&#x4E8E;&#x8303;&#x56F4;&#x7684;RangePartitioner&#x3002;&#x53EA;&#x6709;&#x5BF9;&#x4E8E;key-value&#x7684;RDD&#xFF0C;&#x624D;&#x4F1A;&#x6709;Partitioner&#xFF0C;&#x975E;key-value&#x7684;RDD&#x7684;Parititioner&#x7684;&#x503C;&#x662F;None&#x3002;Partitioner&#x51FD;&#x6570;&#x4E0D;&#x4F46;&#x51B3;&#x5B9A;&#x4E86;RDD&#x672C;&#x8EAB;&#x7684;&#x5206;&#x7247;&#x6570;&#x91CF;&#xFF0C;&#x4E5F;&#x51B3;&#x5B9A;&#x4E86;parent RDD Shuffle&#x8F93;&#x51FA;&#x65F6;&#x7684;&#x5206;&#x7247;&#x6570;&#x91CF;&#x3002;</p>
</li>
<li><p>5)&#x4E00;&#x4E2A;&#x5217;&#x8868;&#xFF0C;&#x5B58;&#x50A8;&#x5B58;&#x53D6;&#x6BCF;&#x4E2A;Partition&#x7684;&#x4F18;&#x5148;&#x4F4D;&#x7F6E;&#xFF08;preferred location&#xFF09;&#x3002;&#x5BF9;&#x4E8E;&#x4E00;&#x4E2A;HDFS&#x6587;&#x4EF6;&#x6765;&#x8BF4;&#xFF0C;&#x8FD9;&#x4E2A;&#x5217;&#x8868;&#x4FDD;&#x5B58;&#x7684;&#x5C31;&#x662F;&#x6BCF;&#x4E2A;Partition&#x6240;&#x5728;&#x7684;&#x5757;&#x7684;&#x4F4D;&#x7F6E;&#x3002;&#x6309;&#x7167;&#x201C;&#x79FB;&#x52A8;&#x6570;&#x636E;&#x4E0D;&#x5982;&#x79FB;&#x52A8;&#x8BA1;&#x7B97;&#x201D;&#x7684;&#x7406;&#x5FF5;&#xFF0C;Spark&#x5728;&#x8FDB;&#x884C;&#x4EFB;&#x52A1;&#x8C03;&#x5EA6;&#x7684;&#x65F6;&#x5019;&#xFF0C;&#x4F1A;&#x5C3D;&#x53EF;&#x80FD;&#x5730;&#x5C06;&#x8BA1;&#x7B97;&#x4EFB;&#x52A1;&#x5206;&#x914D;&#x5230;&#x5176;&#x6240;&#x8981;&#x5904;&#x7406;&#x6570;&#x636E;&#x5757;&#x7684;&#x5B58;&#x50A8;&#x4F4D;&#x7F6E;&#x3002;</p>
</li>
</ul>

                                
                                </section>
                            
    </div>
    <div class="search-results">
        <div class="has-results">
            
            <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
            <ul class="search-results-list"></ul>
            
        </div>
        <div class="no-results">
            
            <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
            
        </div>
    </div>
</div>

                        </div>
                    </div>
                
            </div>

            
                
                <a href="2019-12-30-spark-in-docker.html" class="navigation navigation-prev " aria-label="Previous page: 1.2019-12-30-spark-in-docker">
                    <i class="fa fa-angle-left"></i>
                </a>
                
                
                <a href="2019-12-31-hadoop-hdfs命令别名.html" class="navigation navigation-next " aria-label="Next page: 3.2019-12-31-hadoop-hdfs命令别名">
                    <i class="fa fa-angle-right"></i>
                </a>
                
            
        
    </div>

    <script>
        var gitbook = gitbook || [];
        gitbook.push(function() {
            gitbook.page.hasChanged({"page":{"layout":"post","title":"2.2019-12-30-spark-RDD","categories":["spark","RDD","分布式弹性数据集"],"description":"RDD介绍","keywords":"spark,RDD,分布式弹性数据集","level":"2.1.2","depth":2,"next":{"title":"3.2019-12-31-hadoop-hdfs命令别名","level":"2.1.3","depth":2,"path":"bigdata/2019-12-31-hadoop-hdfs命令别名.md","ref":"bigdata/2019-12-31-hadoop-hdfs命令别名.md","articles":[]},"previous":{"title":"1.2019-12-30-spark-in-docker","level":"2.1.1","depth":2,"path":"bigdata/2019-12-30-spark-in-docker.md","ref":"bigdata/2019-12-30-spark-in-docker.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"bigdata/2019-12-30-spark-RDD.md","mtime":"2019-12-30T07:40:15.872Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2020-08-27T08:51:05.639Z"},"basePath":"..","book":{"language":""}});
        });
    </script>
</div>

        
    <script src="../gitbook/gitbook.js"></script>
    <script src="../gitbook/theme.js"></script>
    
        
        <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-search/search.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
        
    

    </body>
</html>

